/*
 * Copyright (C) 2020 Baidu, Inc. All Rights Reserved.
 */
package com.map.controller;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.net.HttpRequest;
import cn.edu.hfut.dmic.webcollector.net.Proxys;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import cn.edu.hfut.dmic.webcollector.util.RegexRule;
import com.map.util.ArrayUtil;
import com.map.util.ExcelExportUtils;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.Proxy;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class huiben extends BreadthCrawler {

    private final static String crawlPath = "/Users/baidu/Desktop/yuan";

    private Proxys proxies = new Proxys();

    private List<Map> list = new CopyOnWriteArrayList<>();

    RegexRule regexRule = new RegexRule();

    public huiben() {
        super(crawlPath, false);
        proxies.add("114.104.129.85", 4216);
//        proxies.add("117.69.150.177", 4216);
//        proxies.add("111.229.224.145", 8118);
//        proxies.add("114.98.163.246", 30001);
//        proxies.add("60.188.86.139", 3000);
//        proxies.add("60.188.14.118", 3000);
//        proxies.add("121.36.210.88", 8080);
//        proxies.add("60.188.76.104",3000 );
//        proxies.add("60.188.11.148", 3000);
//        proxies.add("117.69.168.193", 4216);
//        proxies.add("117.88.5.118", 3000);
//        proxies.add("60.188.31.33", 3000);
//        proxies.add("60.188.30.129", 3000);
//        proxies.add("60.188.19.206", 3000);
//        proxies.add("60.188.67.165", 3000);
//        proxies.add("60.188.89.100", 3000);
//        proxies.add("119.50.194.79", 80);
//        proxies.add("117.69.180.146", 4216);
//        proxies.add("122.245.246.194", 3000);

        //this.addSeedAndReturn("https://www.huiben.cn/").type("list");
        for (int pageIndex = 1; pageIndex <= 180; pageIndex++) {
            this.addSeed("https://www.huiben.cn/daquan/jieyue/list-" + pageIndex + ".shtml", "list");
//            this.addSeed("https://www.huiben.cn/book/2561.shtml ", "content");
        }

        setThreads(5);
        getConf().setTopN(100);

        //添加正则表达式规则
        //regexRule.addRule("https://.*");
    }

    @Override
    public Page getResponse(CrawlDatum crawlDatum) throws Exception {

        Proxy proxy = proxies.nextRandom();

        try {
            HttpRequest request = new HttpRequest(crawlDatum);
            request.setProxy(proxy);

            Page page = request.responsePage();
            return page;
        } catch (Exception e) {
            System.out.println("连接超时或者连接失败");
            return getResponse(crawlDatum);
        }
    }


    @Override
    public void visit(Page page, CrawlDatums next) {
        String url = "";
        try {
            url = page.url();

            if (page.matchType("list")) {
                Elements select = page.select(".list-box a").removeAttr(".h_dsb h4 a");
                Set<String> href = select.stream()
                        .map(element -> "https://www.huiben.cn" + element.getAllElements().attr("href"))
                        .filter(s -> s.contains("/book"))
                        .collect(Collectors.toSet());

                next.add(href).type("content");
            } else if (page.matchType("content")) {
                // 名称 适合年龄  主题 作者 出版社 和绘本描述

                // 作 者：田村茂 出版社：二十一世纪出版社    语种：中文 被：浏览了 15655 次  借阅了 16950 次   赞了 128 次  共有  条评论
                String authorAndFrom = page.select(".ny_hb_title div p").text();

                String author = authorAndFrom.substring(authorAndFrom.indexOf("作 者：") + 4, authorAndFrom.indexOf("出版社："));
                String from = authorAndFrom.substring(authorAndFrom.indexOf("出版社：") + 4, authorAndFrom.indexOf("语种："));

                String title = "";
                String age = "";

                Elements titleAndAge = page.select(".ny_hb_title div a");

                for (Element element : titleAndAge) {
                    String text = element.text();
                    if (text.contains("岁")) {
                        age = age + "  " + text;
                    } else if (text.contains("条评论")) {
                    } else {
                        title = title + "  " + text;
                    }
                }

                String name = page.select(".ny_hb_title div h3").text();


                String image = "https:" + page.select(".ny_hb_title img").attr("src");

                String desc = "";
                Elements descElements = page.select(".ny_hb_content .neirong p span");

                if (descElements.size() > 2) {
                    desc = page.select(".ny_hb_content .neirong p span").get(2).text();
                }

                String images = "";

                Elements imagesElements = page.select(".ny_hb_content .neirong img");

                for (Element element : imagesElements) {
                    String text = element.attr("src");
                    images = images + ",https://static.huiben.cn" + text;
                }
                Map map = new HashMap();
                map.put("name", name);
                map.put("author", author);
                map.put("from", from);
                map.put("title", title);
                map.put("age", age);
                map.put("image", image);
                map.put("desc", desc);
                map.put("images", images);
                list.add(map);
            }
        } catch (Exception e) {
            System.out.println("URL:" + url);
            System.out.println("本地请求出错" + e);
        }
    }

    public static void main(String[] args) throws Exception {
        //测试
        huiben crawler = new huiben();
        crawler.start(2);

        String[][] arr = ArrayUtil.listToArray(crawler.list);
        String fileName = new String((System.currentTimeMillis() + ".xls").getBytes("UTF-8"), "ISO8859-1");

        File file = new File("/Users/baidu/Desktop/yuan/" + fileName);

        OutputStream os = new FileOutputStream(file);
        ExcelExportUtils.exportFormatExcel(arr, "数据", os);

    }
}